package com.nlp.reptile;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 
 * @author ygsong.abcft
 *
 */
public class Reptile {
	
	ConcurrentLinkedQueue<String> queue = new ConcurrentLinkedQueue<String>();
	BlockingQueue<String[]> data = new LinkedBlockingQueue<String[]>();
	
	private Reptile() {
		//queue.offer("https://www.taobao.com/");
		queue.offer("http://m.womai.com/0c601.shtml");
		queue.offer("http://m.womai.com/0c601.shtml");
	}
	
	private static class Holder {
		private static final Reptile INSTANCE = new Reptile();
	}
	
	public static Reptile getInstance() {
		return Holder.INSTANCE;
	}
	
	public void reptile() throws IOException {
		
		String url = null;
		String html = null;
		String title = null;
		while (queue.size() > 0) {
			url = queue.poll();
			if (url == null || "".equals(url)) {
				continue;
			}
			if (url.startsWith("http") || url.startsWith("https")) {
				//System.out.println(url);
				Document doc = null;
				try {
					 doc= Jsoup.connect(url).get();
				} catch (Exception e) {
					continue;
				}
				html = doc.html();
				//System.out.println(html);
				title = doc.title();
				String[] ss = new String[3];
				ss[0] = title;
				ss[1] = html;
				/*html = removeTag(html);
				html = html.replaceAll("[\\s\\n \\t]+","").replaceAll("\\d+","");*/
				//System.out.println(html);
				Elements meta = doc.select("meta");
				for (Element element : meta) {
					if (element.hasAttr("content")) {
						String content = element.attr("content");
						if (content.contains("charset")) {
							System.out.println(content);
							String[] str = content.split(";");
							for (String s : str) {
								System.out.println(s);
								if (s.contains("=")) {
									ss[2] = s.split("=")[1];

								}
							}
						}
						
					}
					
					String charset = element.attr("charset");
					if (element.hasAttr("charset")) {
						ss[2] = charset;
						break;
					}
				}
				data.offer(ss);

				Elements elements = doc.select("a");
				for (Element element : elements) {
					url = element.attr("href");
					if (!url.startsWith("http")) {
						continue;
					}
					if (!url.startsWith("https")) {
						continue;
					}
					queue.offer(url);
				}
			}
		}
		
	}
	
	public String removeTag(String htmlStr) {
		String regEx_script = "<script[^>]*?>[\\s\\S]*?<\\/script>"; // script
		String regEx_style = "<style[^>]*?>[\\s\\S]*?<\\/style>"; // style
		String regEx_html = "<[^>]+>"; // HTML tag
		String regEx_space = "\\s+|\t|\r|\n";// other characters

		Pattern p_script = Pattern.compile(regEx_script, Pattern.CASE_INSENSITIVE);
		Matcher m_script = p_script.matcher(htmlStr);
		htmlStr = m_script.replaceAll("");
		Pattern p_style = Pattern.compile(regEx_style, Pattern.CASE_INSENSITIVE);
		Matcher m_style = p_style.matcher(htmlStr);
		htmlStr = m_style.replaceAll("");
		Pattern p_html = Pattern.compile(regEx_html, Pattern.CASE_INSENSITIVE);
		Matcher m_html = p_html.matcher(htmlStr);
		htmlStr = m_html.replaceAll("");
		Pattern p_space = Pattern.compile(regEx_space, Pattern.CASE_INSENSITIVE);
		Matcher m_space = p_space.matcher(htmlStr);
		htmlStr = m_space.replaceAll(" ");
		return htmlStr;
	}
	
	public void write() {
		
		while(true) {
			FileOutputStream fos = null;
			FileChannel channel = null;
			try {
				String[] take = data.take();
				String dir = "D:\\Mallet\\test\\womai\\"+take[0];

				fos = new FileOutputStream(dir+".html");
				channel = fos.getChannel();
				String html = take[1];
				ByteBuffer buffer = ByteBuffer.allocate(html.getBytes().length);
				try {
					if (take[2] == null || "".equals(take[2])) {
						take[2] = "UTF-8";
					}
					buffer.put(html.toString().getBytes(take[2]));

				} catch (Exception e) {
					System.out.println(take[2]);
					e.printStackTrace();
				}
				buffer.flip();
				try {
					channel.write(buffer);
				} catch (IOException e) {
					e.printStackTrace();
				}
			} catch (Exception e) {
				e.printStackTrace();
			}finally {
				try {
					if (channel != null) {
						channel.close();
					}
					if (fos != null) {
						fos.close();
					}
				} catch (IOException e) {
					e.printStackTrace();
				}
				
			}
			
		}
	}
	
	public static void main(String[] args) throws IOException {

		CountDownLatch latch = new CountDownLatch(10);
		ExecutorService es = Executors.newFixedThreadPool(10);
		for (int i = 0; i <5; i++) {
			es.execute(()->{
				try {
					Reptile.getInstance().reptile();
				} catch (IOException e) {
					e.printStackTrace();
				}
				latch.countDown();
			});
		}
		
		for (int i = 0; i <5; i++) {
			es.execute(()->{
				Reptile.getInstance().write();
				latch.countDown();
			});
		}
		try {
			latch.await(10,TimeUnit.MINUTES);
			es.shutdown();
		} catch (InterruptedException e) {
			e.printStackTrace();
			es.shutdownNow();
		}
	}

}
